"""
.. classtree:: ProxyUni

.. dotgraph::

    main -> parse -> execute;
    main -> init;
    main -> cleanup;
    execute -> make_string;
    execute -> printf
    init -> make_string;
    main -> printf;
    execute -> compare;

"""

import re
import inspect
import sys
import os
import tempfile
import types
try:
    import external.ply.lex as lex
    import external.ply.yacc as yacc
except ImportError:
    import ply.lex as lex
    import ply.yacc as yacc

from common import capitalize, uncapitalize
import warnings
from arguments import *
from utilitytypes import *


# increase from 0 to 1 or 2 for more debug feedback
def verbose():
    return 0

def currentfn():
    try:
        return sys._getframe(1).f_code.co_name
    except:
        pass


class NameParseError(Exception):
    pass

class ParsingWarning(UserWarning):
    pass


ProxyUni = proxyClass(unicode, 'ProxyUni', module=__name__, dataFuncName='compileName', remove=['__getitem__', '__doc__'])  # 2009 Beta 2.1 has issues with passing classes with __getitem__

# For parsed objects, Token or upper level constructs
class Parsed(ProxyUni):

    _parser = None
    _accepts = ()
    _name = None
    classes = {}

# class Parsed(unicode):
#
#    _parser = None
#    _accepts = ()

    @classmethod
    def accepts(cls, other):
        """ Checks if this Parsed class can accept another object as a subpart without reparsing """
        if isinstance(other, Parsed):
            for t in cls._accepts:
                if t == other.__class__.__name__:
                    return True
        return False

    def compileName(self):
        newname = u''
        partList = []

        def getParts(obj, newname):
            try:
                for x in obj.parts:
                    # print repr(x)
                    newname = getParts(x, newname)
            except AttributeError:
                # print "DEAD", repr(obj)
                newname += unicode(obj._name)
            return newname
        self._name = getParts(self, newname)
        return self._name

    @classmethod
    def getParserClass(cls, parsername):
        pass

    # init class attributes, all objects of a Parsed class share the same parser
    # TODO : check if it can be a problem with multithreading ? In that case we'll need a parser per instance
    @classmethod
    def classparserbuild(cls, **kwargs):
        """ Inits class Parser, all instances of a Parsed class share the same yacc parser object """

        clsname = cls.__name__
        try:
            # class declaration specifies a parser class
            parser = cls._parser
        except:
            # default rule
            parsername = cls.__name__ + "Parser"
            parser = Parser.classes.get(parsername, None)
            cls._parser = parser
            warnings.warn("could not read '_parser' for %s, building Parser name %s from Parsed class name %s" % (cls, parsername, clsname), UserWarning)

        if parser is not None:
            # if parser hasn't been built yet, build it
            if not isinstance(parser, Parser):
                # parser is a class
                if inspect.isclass(parser):
                    parsername = parser.__name__
                    if not issubclass(parser, Parser):
                        raise ValueError, "Parser %s specified in Parsed class %s is not a Parser class" % (parsername, cls.__name__)
                # parser is a string
                elif parser in Parser.classes:
                    parsername = parser
                    parser = Parser.classes[parsername]
                else:
                    raise ValueError, "Invalid Parser specification %r in Parsed class %s" % (parser, cls.__name__)

                # build class Parser, replace class _parser by the Parser instance object

                # print "Building parser instance of Parser %s for Parsed class: %s" % (parser, cls.__name__)
                # replace _parser attribute (which held a Parser class or classname) with parser class instance
                cls._parser = parser()
                cls._parser.build(**kwargs)
                # return cls._parser
        else:
            raise TypeError, "Parsed class %s does not define a parser, check declarations" % cls.__name__

    @classmethod
    def classparse(cls, data, **kwargs):
        clsname = cls.__name__
        data = unicode(data)
        debug = kwargs.get('debug', verbose())
        errmsg = ''
        # print "Calling parser %s with debug %s" % (cls.classparser(), debug)
        result = cls.classparser().parse(data, debug=debug)
        if cls.classparser().errorcount:
            # use error or warning ?
            errmsg = "cannot parse '%s' to a valid %s, %d parser errors" % (data, clsname, cls.classparser().errorcount)
            isValid = False
        elif not isinstance(result, cls):
            # parse successful but returned a different class than exected
            errmsg = "parsing '%s' is valid, but as a %s Parsed object, and not as a %s Parsed object as it was parsed against" % (data, result.__class__.__name__, clsname)
            isValid = False
        elif not result == data:
            # should return a Parsed object with the same string value as the parsed string
            errmsg = "parsing '%s' raised no error, but the resulting name is %s is different from the nput string %s " % (result, data)
            isValid = False
        else:
            # parse successful
            isValid = True

        # check for error in parsing and correct and raise a warning or raise an error
        # TODO : corrections and error handling
        if not isValid:
            # can try to auto-correct some badly formed names
            raise NameParseError, errmsg

        # position is set to position of first found Parsed object
        if (result.sub):
            result._pos = result.sub[0].pos
        else:
            result._pos = 0

        result._valid = True
        return result

    @classmethod
    def classparser(cls):
        """ parser object for that class """
        return cls._parser

    # instance methods

    def parse(self, data, **kwargs):
        return self.__class__.classparse(data, **kwargs)

    @property
    def parser(self):
        """ parser object for that class """
        return self.__class__.classparser()

    @property
    def tokens(self):
        """ iterates self as leaf level lexed tokens """
        for i in expandArgs(self._sub):
            if isinstance(i, Token):
                yield i

    @property
    def sub(self):
        """ Internally stored parsing data for this Parsed object sub parts """
        return self._sub

    def setSubItem(self, index, value):
        """ Change the value of one of the Parsed sub parts.  The new value will first be parsed as the same
        type as it is replacing."""
        cls = self._sub[index].__class__
        sublist = list(self._sub)
        sublist[index] = cls(value)
        self._sub = tuple(sublist)

    @property
    def pos(self):
        """ position of that Parsed object """
        return self._pos

    def isValid(self):
        """ Validity """
        return self._valid

    def copy(self):
        """return an new independent copy of the parsed object"""
        return self.__class__(self._sub)

    def findType(self, type):
        res = []
        for x in self.sub:
            if isinstance(x, type):
                res.append(x)
            else:
                res += x.findType(type)
        return res

    def __new__(cls, *args, **kwargs):
        """ Creation of a Parsed object from a LexToken, other Parsed of compatible type or string,
            if a string is passed it will be parsed and checked for compatibility with this Parsed type """

        debug = kwargs.get('debug', verbose())
        # type checking
        data = None
        if args:
            if len(args) == 1:
                data = args[0]
            else:
                data = tuple(args)

        # some data (when initializing from single arg) can define the type of Parsed object to be created
        ptype = None
        if data is None:
            # only authorize Empty to be built without arguments
            ptype = 'Empty'
        elif isinstance(data, lex.LexToken):
            ptype = kwargs.get('type', data.type)
        elif isinstance(data, Parsed):
            ptype = data.__class__
        # can override type with the keyword 'type'
        ptype = kwargs.get('type', ptype)

        if (cls is Parsed or cls is Token):  # issubclass(cls, Token) ):
            if ptype is not None:
                if verbose():
                    print "__new__ called on Parsed/Token %s with type %r" % (cls.__name__, ptype)
                newcls = Parsed.classes.get(ptype, None)
                # can only specify an existing subclass of cls
                if newcls is None:
                    raise TypeError, "Type %s does not correspond to any existing Parsed sub-class (%s does not exist)" % (ptype, cls.__name__)
                else:
                    clsname = newcls.__name__
                if not issubclass(newcls, cls):
                    raise TypeError, "Type %s would create a class %s that is not a sub-class of the class %s that __new__ was called on" % (ptype, clsname, cls.__name__)
            else:
                raise TypeError, "Class %s is an abstract class and can't be created directly, you must specify a valid sub-type" % (cls.__name__)
        else:
            if verbose():
                print "__new__ called on explicit class %s" % (cls.__name__)
            clsname = cls.__name__
            newcls = cls

        # print "Creating new instance of Parsed class: %r" % newcls

        # process arguments and build, check arguments compatibility with that type
        pos = None
        sub = []
        valid = False
        value = data

        # if debug : print "VALUE1", value, repr(value)

        # special case for LexToken
        if isinstance(data, lex.LexToken):
            if issubclass(newcls, Token):
                # from a unique lex Token, do not check if also creating a Token
                sub = []
                pos = data.lexpos
                value = data.value
                valid = True
            else:
                # build a Token from it
                value = Token(data.value, ptype=data.type, pos=data.pos)

        if data is None:
            # Tokens can have default value to allow initialization without arguments
            try:
                value = newcls.default()
                valid = True
            except:
                valid = False
        elif isinstance(data, newcls):
            # if debug : print "IS INSTANCE", data, repr(data)
            # from a similar class, copy it
            sub = data.sub
            pos = data.pos
            valid = data.isValid()
            value = unicode(data)
        elif newcls.accepts(data):
            # if debug : print "ACCEPTS", data, repr(data)
            # from a compatible Parsed sub class, build the sub list from it
            sub.append(data)
            pos = data.pos
            valid = data.isValid()
            value = unicode(data)
        elif isSequence(data) and not isinstance(data, basestring):
            # building from sub parts, must be of the same type and in class _accepts
            # TODO : use yacc own rules for accepts
            if data:
                valid = True
                p = 0
                for arg in data:
                    # converts LexTokens directly
                    if isinstance(arg, lex.LexToken):
                        a = Token(arg.value, ptype=arg.type, pos=data.pos)
                    else:
                        a = arg
                    # now check if it's a suitable sub-part or derived class
                    if isinstance(a, newcls):
                        sub += a.sub
                    elif newcls.accepts(a):
                        sub.append(a)
                    else:
                        valid = False
                        break
                value = u"".join(map(unicode, data))
                if valid:
                    pos = sub[0].pos
                else:
                    sub = []
            else:
                value = ''
        else:
            if debug:
                print "REPARSE", data, repr(data)
            # reparse unless it's a Token we already know the type of
            value = unicode(data)
            if issubclass(newcls, Token) and newcls is not Token:
                sub = []
                pos = 0
                valid = True
            else:
                valid = False

        # parse if necessary
        if valid:
            # print "No reparsing necessary for a resulting value %s (%r)" % (value, value)
            strvalue = unicode(value)
        elif isinstance(value, basestring):
            if debug:
                print "%s: Will need to reparse value %r" % (clsname, value)
            newcls.classparserbuild(debug=debug)
            if debug:
                print "VALUE", value, type(value)
            result = newcls.classparse(value, debug=debug)
            if debug:
                print "RESULT", result, type(result), isinstance(result, newcls)
            if result is not None and isinstance(result, newcls):
                strvalue = unicode(result)
                valid = result._valid
                sub = result._sub
                pos = result._pos
                if debug:
                    print "SUB", sub
            else:
                strvalue = ''
                valid = False
        else:
            raise TypeError, "invalid argument(s) %r, cannot be parsed to create a Parsed object of type %s" % (value, clsname)

        if valid:
            # create a unicode object with appropriate string value
            newobj = super(Parsed, cls).__new__(newcls)
            newobj._name = strvalue
            # if debug: print "NAME", newobj, type(newobj), sub#, inspect.getmro(newobj)
            # set instance attributes
            newobj._sub = tuple(sub)
            newobj._valid = valid
            # override for pos
            pos = kwargs.get('pos', pos)
            if pos is not None:
                pos += kwargs.get('offset', 0)

            if pos is None or (isinstance(pos, int) and pos >= 0):
                newobj._pos = pos
            else:
                raise ValueError, "A Parsed pos can only be None or an unsigned int, %r invalid" % pos

        return newobj

    def __add__(self, other):
        """ p1.__add__(p2) <==> p1+p2
            if p1 and p2 are of the same Parsed type, it's equivalent to reparsing str(p1) + str(p2)
            if p2 is an accepted sub part of p1, it adds it to the sub-parts
        """
        # The Parsed _accepts defines validity
        # TODO : use yacc own rules to check validity without a full reparse
        cls = self.__class__
        selfvalid = self.isValid()
        sublist = list(self.sub)
        value = unicode(self)
        # check other's type
        if isinstance(other, cls):
            othervalid = other.isValid()
            sublist += other.sub
        elif self.accepts(other):
            othervalid = other.isValid()
            sublist.append(other)
        elif isinstance(other, basestring):
            othervalid = False
        else:
            raise TypeError, "cannot add %s and %s" % (type(self), type(other))

        if selfvalid and othervalid:
            # no reparse
            result = cls(*sublist)
        else:
            # reparse
            result = cls(unicode(self) + unicode(other))

        return result

    def __repr__(self):
        return u"%s('%s', %s)" % (self.__class__.__name__, self, self.pos)

# Parsers, all parser must derive from the Parser class

class Parser(object):

    """ Abstract Base class for all name parsers """
    classes = {}

    def __new__(cls, *args, **kwargs):
        # this class is an abstract base class for all Parser classes, cannot be built directly

        if cls is Parser:
            # type argument can be the name of a Parser class or an instance of the class
            ptype = kwargs.get('type', None)
            if ptype is None:
                raise TypeError, "must specify a Parser class"
            elif isinstance(ptype, Parser) and not ptype is Parser:
                parsercls = ptype
            elif ptype in Parser.classes:
                parsercls = Parser.classes[ptype]
            else:
                raise TypeError, "invalid Parser type: %s" % ptype
        else:
            # subclasses of Parser
            parsercls = cls

#        if 'tokensDict' in parsercls.__dict__ and 'rulesDict' in parsercls.__dict__:
#            rulesDict = parsercls.rulesDict
#            tokensDict = parsercls.tokensDict
#        else:

        reserved = []
#        #reserved = [ k for k,v in parsercls.tokensDict.items() if not k.startswith( 't_') ]
#        if hasattr( parsercls, '_reserved'):
#            for k, v in parsercls._reserved.items():
#                # reserved should be a map of reserved names (as used in the code to parse) to token names (as used in PLY).
#                # as such, we only care about the PLY names, ie. the values
#                reserved.append(v)

        rulesDict, tokensDict = Parser.getRulesAndTokens(parsercls)

        tokens = tokensDict.keys()
        rules = list(rulesDict.keys())
        # Sort them by line number of declaration as it's how the yacc builder works to order rules
        # TODO : some more explicit rule order handling (when precedence isn't an option) ?
        rules.sort(lambda x, y: cmp(rulesDict[x].func_code.co_firstlineno, rulesDict[y].func_code.co_firstlineno))
        # print "sorted rules:", [(r, parsercls.rulesDict[r].func_code.co_firstlineno) for r in rules]
        rules = tuple(rules)

        parsercls.tokens = tuple(tokens + list(set(reserved)))
        parsercls.rules = rules

        # print "%s tokens %s" % ( parsercls, parsercls.tokens )
        # class must have a start attribute for __init__, start can be None though
        # must not inherit start as parsed would not parse own class new rules
        if not 'start' in parsercls.__dict__:
            parsercls.start = None

        # TODO : same for precedence rules
        return super(Parser, cls).__new__(parsercls, *args, **kwargs)

    def __init__(self, *args, **kwargs):
        self.errorcount = 0
        self.lexer = None
        self.parser = None

    def t_error(self, t):
        warnings.warn("illegal character in '%s' at %i: '%s'" % (t.lexer.lexdata, t.lexpos, t.value[0]), ParsingWarning, stacklevel=1)
        self.errorcount += 1
        t.lexer.skip(1)

    def p_error(self, p):
        print "error token", p
        if p is None:
            warnings.warn("unexpected end of file", ParsingWarning, stacklevel=1)
        else:
            warnings.warn("syntax error in '%s' at %i: '%s'" % (p.lexer.lexdata, p.lexpos, p.value), ParsingWarning, stacklevel=1)
        self.errorcount += 1

        # Just discard the token and tell the parser it's okay.
        # yacc.errok()
        # yacc.errok(). This resets the parser state so it doesn't think it's in error-recovery mode. This will prevent an error token from being generated and will reset the internal error counters so that the next syntax error will call p_error() again.
        # yacc.token(). This returns the next token on the input stream.
        # yacc.restart(). This discards the entire parsing stack and resets the parser to its initial state.

    @staticmethod
    def getRulesAndTokens(parsercls):
        """
        build the tokens and precedence tuples from inherited declarations.
        gather tokens and rules definition from Parser class members (own and inherited)
        """

        tokensDict = {}
        rulesDict = {}
        for name, obj in inspect.getmembers(parsercls):
            if name.startswith('t_') and name != 't_error':

                if isinstance(obj, basestring):
                    v = obj
                elif inspect.isfunction(obj) or inspect.ismethod(obj):
                    v = obj.__doc__
                else:
                    raise SyntaxError, "Token definition %s defines neither a string nor a function, unable to parse" % m[0]
                k = name[2:]
                tokensDict[k] = obj
            elif name.startswith('p_') and inspect.ismethod(obj) and name != 'p_error':
                k = name[2:]
                rulesDict[k] = obj
#            elif name == '_reserved' and isinstance(obj,dict):
#                # reserved attribute holds a list of reserved token keywords.
#                # these should only exist on
#                print "FOUND reserved!"
#                for k,v in obj.items():
#                    tokensDict[v]=None

        return rulesDict, tokensDict

    def build(self, **kwargs):
        debug = kwargs.get('debug', verbose())
        start = kwargs.get('start', self.__class__.start)
        parserspath = kwargs.get('outputdir', tempfile.gettempdir())
        if debug:
            print "nameparse parsers path", parserspath
        method = kwargs.get('method', 'LALR')
        if debug:
            print "Build for", self.__class__.__name__
            print "tokens:"
            for t in self.__class__.tokens:
                # t is not always in tokensDict, not sure if this is bad or not...
                # print "\t%s = %r" % (t, self.__class__.tokensDict[t])
                print "\t%s" % (t)
            print "rules:"
            for t in self.__class__.rules:
                # t is not always in rulesDict, not sure if this is bad or not...
                # print "\t%s = %r" % (t, self.__class__.rulesDict[t].__doc__)
                print "\t%s" % (t)
            print "start: %s" % start

        if self.lexer is None:
            lextab = self.__class__.__name__ + "_lex"
            lkwargs = {'debug': debug, 'lextab': lextab}
            self.lexer = lex.lex(object=self, **lkwargs)
        if self.parser is None:
            tabmodule = self.__class__.__name__ + "_yacc_" + start
            pkwargs = {'outputdir': parserspath, 'debug': debug, 'tabmodule': tabmodule, 'start': start, 'method': method}
            self.parser = yacc.yacc(module=self, **pkwargs)

    def parse(self, data, **kwargs):
        self.errorcount = 0
        return self.parser.parse(data, lexer=self.lexer, **kwargs)

class Token(Parsed):

    """ A class for token types, allows direct initialization from a string and type without checking
        to avoid unnecessary double parse of the string """
    pass


# token parser, can directly use re
class TokenParser(Parser):

    """ Abstract base class for Token parser """
    _pattern = None
    _type = None

    def build(self, **kwargs):
        pattern = kwargs.get('pattern', self.__class__._pattern)
        try:
            self.parser = re.compile(pattern)
        except:
            raise ValueError, "cannot build Token Parser from pattern %r. you must set the _pattern attribute to a valid regular expression" % pattern

    def parse(self, data, **kwargs):
        self.errorcount = 0
        if self.parser.match(data) is not None:
            return Token(data, type=self._type, pos=0)
        else:
            warnings.warn("%s is not matching %s pattern %r" % (data, self.__class__.__name__, self._pattern))
            self.errorcount += 1

# special purpose empty parser
class EmptyTokenParser(Parser):

    def build(self, **kwargs):
        pass

    def parse(self, data, **kwargs):
        self.errorcount = 0
        if data:
            self.errorcount = 1
        else:
            return Empty()

# derived TokenParser classes will be built for every token definition detected in Parser classes in this module

class EmptyParser(Parser):

    """ Parser for the empty production """

    start = 'Empty'

    def p_empty(self, p):
        'Empty : '
        pass

def isParsedClass(x):
    try:
        return issubclass(x, Parsed)
    except:
        return False
def isParserClass(x):
    try:
        return issubclass(x, Parser)
    except:
        return False

def _printClassTree(cls):
    def _printTree(data, level):
        if isinstance(data, tuple):
            level += 1
            for d in data:
                _printTree(d, level)
        else:
            print data

    print cls
    tree = inspect.getclasstree(cls)
    print tree
    _printTree(tree)


class autoparsed(type):

    """metaclass for dramatically reducing setup syntax for simple hierarchies"""
    def __new__(mcl, classname, bases, classdict):

        newcls = super(autoparsed, mcl).__new__(mcl, classname, bases, classdict)
        module = __import__(newcls.__module__, globals(), locals(), [''])
        # print "autoparsed", newcls

        if issubclass(newcls, Parsed):
            # find the attribute _parser, which holds the parser class for this Parsed type
            parsercls = classdict.get('_parser', None)

            if isParserClass(parsercls) or (parsercls is None and '_rules' in classdict):

                rules = classdict.get('_rules', None)
                if rules is not None:
                    # there's no parser, but there is a set of rules. use this to create a default parser
                    if not isIterable(rules):
                        raise TypeError, "_rules attribute must be an iterable list of string rules"

                    # first add the rules to the parserdict as attributes, these will be converted to methods, below
                    parserdict = {}
                    for i, rule in enumerate(rules):
                        attrname = 'p_%s_%02d' % (classname, i + 1)
                        parserdict[attrname] = rule
                else:
                    # parser already exists. use it, but use __dict__, not inherited info
                    parserdict = parsercls.__dict__

                # a dictionary of updates to be made to the parser
                updatedict = {}

                # automatically set the start attribute to the name of this class
                if 'start' not in parserdict:
                    updatedict['start'] = classname

                # gather up all the tokens and rules and store them separately
                tokensDict = {}
                rulesDict = {}
                reservedDict = {}
                accepts_set = set(classdict.get('_accepts', []))

                classNameReg = re.compile('[a-zA-Z][a-zA-Z0-9]*')
                for name, obj in parserdict.items():
                    # print "class %s has attribute %s" % (parsercls.__name__, m)

                    # RULES
                    if name.startswith('p_'):
                        shortname = name[2:]
                        # shorthand syntax: needs to be converted to a function
                        if isinstance(obj, basestring):
                            v = obj
                            needsWrapping = True
                        # regular syntax: already a function
                        elif inspect.isfunction(obj) or inspect.ismethod(obj):
                            v = inspect.getdoc(obj)
                            needsWrapping = False
                        else:
                            raise SyntaxError, "Token definition %s defines neither a string nor a function, unable to parse" % name

                        # process the docstring
                        accepts = []
                        colonSplit = [x.strip() for x in v.split(':')]
                        if len(colonSplit) == 1:
                            v = classname + ' : ' + v
                            colonSplit = [classname, colonSplit[0]]

                        if len(colonSplit) == 2 and colonSplit[0] != '':
                            pipeSplit = [x.strip() for x in colonSplit[1].split('|')]
                            for grp in pipeSplit:
                                # be sure to filter out ourselves
                                accepts += [x for x in grp.split() if classNameReg.match(x) and x != classname]

                        # add the classes involved in the rules
                        accepts_set.update(accepts)

                        # create the function
                        if needsWrapping:
                            def p_func(self, p):
                                p[0] = newcls(*p[1:len(p)])
                            p_func.__name__ = name
                            p_func.__doc__ = v
                            if verbose():
                                print "overwriting %s._parser.%s with yacc function: %r" % (classname, name, v)
                            updatedict[name] = p_func
                            rulesDict[name] = p_func
    #                        k = m[0][2:]
    #                        if k in tokensDict :
    #                            warnings.warn("Token %s redefined in Parser %s" % (k, parser), UserWarning)
    #                        tokensDict[k] = v
                        else:
                            rulesDict[name] = obj

                    # TOKENS
                    elif name.startswith('t_') and name != 't_error':
                        shortname = name[2:]
                        if isinstance(obj, basestring):
                            v = obj
                        elif inspect.isfunction(obj) or inspect.ismethod(obj):
                            v = obj.__doc__
                        else:
                            raise SyntaxError, "Token definition %s defines neither a string nor a function, unable to parse" % name
                        tokensDict[name] = obj

                if verbose():
                    print "%s accepts: %s" % (newcls.__name__, list(accepts_set))

                # find bases
                bases = []
                for parsedname in accepts_set:
                    try:
                        parsedcls = getattr(module, parsedname)
                    except AttributeError:
                        warnings.warn("Could not find a Parsed class %s in module %s.  Assuming that it is an custom token." % (parsedname, module), UserWarning)
                        continue

                    try:
                        parser = parsedcls._parser
                    except AttributeError:
                        try:
                            parser = getattr(module, parsedname + 'Parser')
                        except AttributeError:
                            raise SyntaxError, "Cannot find parser class for parsed class %s. It should be explicitly or automatically created as %s._parser or be named %sParser" % (parsedname, parsedname, parsedname)

                    assert isinstance(parser, Parser) or (inspect.isclass(parser) and issubclass(parser, Parser)), '%s._parser is not a subclass of Parser, it is %r' % (parsedname, parser)

                    bases.append(parser)

                # some of our bases might be subclasses of other bases.  we want the smallest list of superclasses, so weed out
                # other classes that are directly dependent on other bases
                newbases = []
                for i in bases:
                    issuperclass = False
                    #parents = inspect.getmro(i)
                    for j in bases:
                        if i in inspect.getmro(j)[1:]:
                            issuperclass = True
                            break
                    if not issuperclass:
                        newbases.append(i)

                newbases = sorted(newbases, key=lambda x: len(inspect.getmro(x)), reverse=True)

                # print "%s bases" % newcls.__name__

                for base in newbases:
                    # print base
                    # print '\t', base.tokensDict
                    # print '\t', base.rulesDict
                    tokensDict.update(base.tokensDict)
                    rulesDict.update(base.rulesDict)
#                    if hasattr(base, '_reserved'):
#                        reservedDict.update( base._reserved )

                updatedict.update(tokensDict)
                updatedict.update(rulesDict)

                # for i, parent in enumerate(inspect.getmro(base)):
                #    print '\t'*i, parent
                # print inspect.getclasstree(bases)
#                    for base in newbases:
#                        _printClassTree( base )

                if rules is not None:
                    # time to make the parser
                    # we use the Rules in the 'accepts' list to determine the bases of this parser class

                    # TODO : filter out bases that are superclasses of other bases: we only need one set

                    parserName = classname + 'Parser'
                    #newbases = sorted( newbases, key = lambda x: x.__name__)

                    updatedict['tokensDict'] = tokensDict
                    updatedict['rulesDict'] = rulesDict
#                    updatedict['_reserved'] = reservedDict

                    parserdict.update(updatedict)

                    # create parser class
                    parsercls = type(parserName, (Parser,), parserdict)
                    parsercls.__module__ = module.__name__
                    setattr(newcls, '_parser', parsercls)
                else:
                    # update the existing parser
                    for k, v in updatedict.iteritems():
                        setattr(parsercls, k, v)

                setattr(newcls, '_accepts', tuple(accepts_set))

            elif issubclass(newcls, Token):
                # a token subclass defines its regex on the '_token' attr.
                # if this attr is defined, it causes a Parser class to be created for this token, along with
                # a rule, which is named after the class
                token = classdict.get('_token', None)
                if token is None:
                    raise TypeError, "Token classes that use the autoparsed metaclass must define a _token attribute"

                # print "automatically setting up token class for %s" % newcls.__name__

                tokenBasename = classname + '_Token'
                tokenName = 't_' + tokenBasename
                ruleName = 'p_' + tokenBasename
                parserName = classname + 'Parser'

                if hasattr(token, '__name__'):
                    # print "setting token name to %s" % tokenName
                    token.__name__ = tokenName

                parserdict = {}
                parserdict[tokenName] = token

                tokensDict = {tokenName: token}

#                if hasattr(newcls, '_reserved'):
#                    print "adding _reserved to %s" % parserName
#                    parserdict['_reserved'] = newcls._reserved

                parserdict['tokensDict'] = tokensDict

                # the rule is named after the class
                doc = '''%s : %s ''' % ( classname, tokenBasename )
                #class _parser( Parser ): pass

                def rule(self, p):
                    p[0] = newcls(p[1], pos=p.lexpos(1))
                rule.__name__ = ruleName
                rule.__doc__ = doc
                parserdict[ruleName] = rule
                parserdict['rulesDict'] = {ruleName: rule}

                # create the Parser class
                print "creating parser %s" % parserName
                parsercls = type(parserName, (Parser,), parserdict)
                parsercls.__module__ = module.__name__
                #setattr( _parser, tokenName, token )
                #setattr( _parser, ruleName, rule )
                setattr(newcls, '_parser', parsercls)

        return newcls

def _getTokenPatterns(parsercls):
    tokensDict = {}
    for name, obj in parsercls.__dict__.items():
        # print "class %s has attribute %s" % (parsercls.__name__, m)
        if name.startswith('t_') and name != 't_error':
            # strip off 't_'
            k = name[2:]
            if isinstance(obj, basestring):
                v = obj
            elif inspect.isfunction(obj) or inspect.ismethod(obj):
                v = obj.__doc__
            else:
                raise SyntaxError, "Token definition %s defines neither a string nor a function, unable to parse" % name
            k = name[2:]
            if k in tokensDict:
                warnings.warn("Token %s redefined in Parser %s" % (k, parser), UserWarning)
            tokensDict[k] = v
    return tokensDict

# do it
#_addedTokenClasses =_createTokenClasses(debug=verbose())

# Build a dict of all existing Parser and Parsed classes in this module
# class Parsed.classes(dict) :
#    __metaclass__ =  metaStatic


# def parsedClasses(module):
#    return dict(inspect.getmembers(module, isParsedClass))
# Stores it at import so that the inspect method isn't recalled at each query
# Parsed.classes(Parsed.classes)

# class Parser.classes(dict) :
#    __metaclass__ =  metaStatic


Parsed.classes = {}
Parser.classes = {}

# def parserClasses(module):
#    return dict(inspect.getmembers(module, isParserClass))
# Stores it at import so that the inspect method isn't recalled at each query
# Parser.classes(Parser.classes)

def process(module=None):
    """cache out a dictionary of all Parsed and Parser classes, and create token classes"""
    if module:
        # User supplied a module object.
        if isinstance(module, types.ModuleType):
            module_dict = module.__dict__
#        elif isinstance(module, _INSTANCETYPE):
#            _items = [(k,getattr(module,k)) for k in dir(module)]
#            ldict = { }
#            for i in _items:
#                ldict[i[0]] = i[1]
        else:
            raise ValueError, "Expected a module"

    else:
        # No module given.  We might be able to get information from the caller.
        # Throw an exception and unwind the traceback to get the globals

        try:
            raise RuntimeError
        except RuntimeError:
            e, b, t = sys.exc_info()
            f = t.tb_frame
            f = f.f_back           # Walk out to our calling function
            module_dict = f.f_globals    # Grab its globals dictionary

    #_createTokenClasses(ldict, debug=verbose() )

    parserDict = {}
    parsedDict = {}
    tokensDict = {}

    # gather up all tokens, parsers, and parsed
    for name, obj in module_dict.iteritems():
        if isParserClass(obj):
            parserDict[name] = obj
            tokensDict.update(_getTokenPatterns(obj))

        # find Parsers that are stored on Parsed classes as _parser
        elif isParsedClass(obj):
            parsedDict[name] = obj
            if hasattr(obj, '_parser'):
                if isParserClass(obj._parser):
                    tokensDict.update(_getTokenPatterns(obj._parser))

    for token in tokensDict:
        pattern = tokensDict[token]
        parsedName = token
        parserName = token + "Parser"

        if verbose():
            print "adding class %s for token %s of pattern r'%s'" % (parserName, token, pattern)

        class ThisTokenParser(TokenParser):

            """ Token Parser stub class """
        # set the Token Parser class attributes
        ThisTokenParser.__name__ = parserName
        #ThisTokenParser.__doc__ = "Parser for token %s=%r" % (token, pattern)
        ThisTokenParser.__module__ = __name__
        ThisTokenParser._pattern = pattern
        ThisTokenParser._type = token
        # add to the module
        module_dict[parserName] = ThisTokenParser
        parserDict[parserName] = ThisTokenParser

        if verbose():
            print "adding class %s for token %s of pattern r'%s'" % (parsedName, token, pattern)

        class ThisToken(Token):

            """ Token stub class """
        # set the Token class attributes
        ThisToken.__name__ = parsedName
        # ThisToken.__doc__ = "Parser for token %s=%r" % (token, pattern)
        ThisToken.__module__ = __name__
        ThisToken._parser = ThisTokenParser
        # add to the module
        module_dict[parsedName] = ThisToken
        parsedDict[parsedName] = ThisToken

    Parser.classes = parserDict
    Parsed.classes = parsedDict
